[ { "memory": 3058188544, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3060547840, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3068477696, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3076407552, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3077095680, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3085025536, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3087384832, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3095314688, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3103244544, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3111174400, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3119104256, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3119792384, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3120480512, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3128410368, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3136340224, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3144270080, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3144958208, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3145646336, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3153576192, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3154264320, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3156623616, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3164553472, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3172483328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3180413184, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3182772480, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3190702336, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3191390464, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3192078592, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3200008448, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3207938304, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3215868160, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3216556288, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3217244416, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3225174272, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3227533568, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3235463424, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3237822720, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3245752576, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3248111872, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3256041728, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3256729856, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3264659712, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3272589568, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3280519424, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3281207552, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3281895680, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3284254976, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3284943104, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3292872960, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3295232256, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3303162112, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3303850240, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3306209536, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3308568832, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3316498688, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3318857984, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3326787840, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3334717696, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3335405824, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3336093952, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3344023808, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3351953664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3359883520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3360571648, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3362930944, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3370860800, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3378790656, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3386720512, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3394650368, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3402580224, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3403268352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3403956480, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3411886336, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3414245632, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3416604928, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3417293056, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3425222912, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3427582208, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3429941504, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3437871360, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3445801216, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3448160512, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3448848640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3451207936, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3451896064, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3459825920, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3462185216, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3470115072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3478044928, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3480404224, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3481092352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3483451648, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3491381504, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3492069632, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3499999488, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3500687616, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3508617472, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3509305600, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3509993728, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3517923584, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3520282880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3520971008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3528900864, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3536830720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3539190016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3539878144, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3540566272, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3541254400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3543613696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3545972992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3548332288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3550691584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3558621440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3560980736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3563340032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3565699328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3566387456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3567075584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3569434880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3570123008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3572482304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3573170432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3575529728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3583459584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3585818880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3588178176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3590537472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3592896768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3600826624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3601514752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3609444608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3611803904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3612492032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3614851328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3617210624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3617898752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3620258048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3620946176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3621634304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3622322432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3624681728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3632611584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3634970880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3637330176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3639689472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3647619328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3655549184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3663479040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3671408896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3679338752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3680026880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3687956736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3695886592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3698245888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3706175744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3714105600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3716464896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3724394752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3725082880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3727442176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3728130304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3728818432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3736748288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3744678144, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3752608000, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3760537856, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3768467712, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3770827008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3771515136, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3772203264, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3772891392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3780821248, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3783180544, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3791110400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3799040256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3806970112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3807658240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3808346368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3816276224, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3824206080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3824894208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3832824064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3840753920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3843113216, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3845472512, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3853402368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3854090496, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3854778624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3862708480, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3870638336, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3878568192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3879256320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3887186176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3895116032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3903045888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3910975744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3918905600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3926835456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3929194752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3929882880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3932242176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3934601472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3936960768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3944890624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3947249920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3947938048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3948626176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3950985472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3958915328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3959603456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3961962752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3962650880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3963339008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3971268864, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3979198720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3981558016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3983917312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3984605440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3992535296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3994894592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3997253888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4005183744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4005871872, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4013801728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4014489856, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4015177984, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4017537280, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4025467136, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4027826432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4030185728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4032545024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4040474880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4048404736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4056334592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4064264448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4072194304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4080124160, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4080812288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4088742144, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4096672000, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4099031296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4101390592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4109320448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4111679744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4114039040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4114727168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4122657024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4130586880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4138516736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4146446592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4154376448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4162306304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4170236160, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4178166016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4180525312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4181213440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4189143296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4197073152, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4205003008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4207362304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4209721600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4210409728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4218339584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4226269440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 4234199296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4242129152, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4250059008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4252418304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4260348160, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4262707456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4265066752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4267426048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4275355904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4276044032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4283973888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4291903744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4299833600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4302192896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4302881024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4303569152, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4304257280, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4312187136, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4320116992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4320805120, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4321493248, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4329423104, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4337352960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4345282816, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4353212672, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4355571968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4356260096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4364189952, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4366549248, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4367237376, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4375167232, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4377526528, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4385456384, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4393386240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4394074368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4402004224, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4409934080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4412293376, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4412981504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4415340800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4416028928, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4423958784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4426318080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4427006208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4434936064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4437295360, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4437983488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4445913344, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4448272640, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4450631936, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4452991232, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4453679360, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4454367488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4462297344, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4470227200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4478157056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4480516352, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4481204480, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4481892608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4482580736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4484940032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4492869888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4500799744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4503159040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4503847168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4506206464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4508565760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4516495616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4524425472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4532355328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4534714624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4537073920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4539433216, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4541792512, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4549722368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4557652224, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4565582080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4573511936, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4581441792, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4589371648, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4597301504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4597989632, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4605919488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4608278784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4608966912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4609655040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4610343168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4612702464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4620632320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4621320448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4623679744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4624367872, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4632297728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4634657024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4637016320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4644946176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4647305472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4655235328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4663165184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4671095040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4671783168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4672471296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4673159424, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4673847552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4674535680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4682465536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4683153664, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4685512960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4693442816, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4701372672, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4709302528, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4711661824, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4714021120, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4714709248, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4717068544, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4724998400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4727357696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4735287552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4735975680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4743905536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4746264832, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4746952960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4749312256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4757242112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4759601408, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4761960704, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4769890560, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4777820416, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4778508544, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4780867840, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4781555968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4782244096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4790173952, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4790862080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4798791936, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4799480064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4801839360, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4802527488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4810457344, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4818387200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4819075328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4827005184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4829364480, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4830052608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4832411904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4840341760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4841029888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4841718016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4844077312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4844765440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4845453568, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4846141696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4846829824, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4854759680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4862689536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4863377664, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4865736960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4868096256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4868784384, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4876714240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4877402368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4885332224, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4887691520, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4895621376, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4897980672, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4900339968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4901028096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4901716224, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4904075520, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4904763648, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4907122944, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4915052800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4917412096, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4925341952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4933271808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4941201664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4949131520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4957061376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4964991232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4965679360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4966367488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4967055616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4967743744, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4970103040, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4970791168, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4978721024, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4 } }, { "memory": 4986650880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4994580736, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4995268864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4997628160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4999987456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5007917312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5015847168, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5023777024, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5026136320, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5034066176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5041996032, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5049925888, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5057855744, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5065785600, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5066473728, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5068833024, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5076762880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5084692736, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5092622592, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5100552448, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5108482304, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5116412160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5117100288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5117788416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5125718272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5133648128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5141577984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5149507840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5157437696, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5165367552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5173297408, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5181227264, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5189157120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5191516416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5193875712, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5194563840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5196923136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5197611264, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5205541120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5206229248, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5208588544, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5210947840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5211635968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5219565824, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5227495680, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5235425536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5243355392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5251285248, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5259215104, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5267144960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5275074816, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5277434112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5285363968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5293293824, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5293981952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5294670080, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5295358208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5303288064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5311217920, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5313577216, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5315936512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5318295808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5320655104, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5321343232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5322031360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5322719488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5330649344, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5333008640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5335367936, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5343297792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5351227648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5359157504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5367087360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5375017216, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5382947072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5385306368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5393236224, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5401166080, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5403525376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5411455232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5412143360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5420073216, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5428003072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5435932928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5443862784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5444550912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5452480768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5454840064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5455528192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5463458048, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5471387904, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5479317760, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5481677056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5489606912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5497536768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5505466624, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5507825920, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5515755776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5523685632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5526044928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5533974784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5541904640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5549834496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5557764352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5560123648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5568053504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5570412800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5572772096, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5580701952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5588631808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5590991104, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5591679232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5599609088, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5607538944, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5615468800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5623398656, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5631328512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5632016640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5639946496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5640634624, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5648564480, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5656494336, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5664424192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5672354048, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5673042176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5673730304, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5674418432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5682348288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5684707584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5692637440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5693325568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5701255424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5701943552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5702631680, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5710561536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5718491392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5720850688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5723209984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5731139840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5733499136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5735858432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5743788288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5751718144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5759648000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5760336128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5762695424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5763383552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5765742848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5766430976, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5774360832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5776720128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5777408256, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5779767552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5782126848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5790056704, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5790744832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5793104128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5793792256, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5794480384, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5802410240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5804769536, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5812699392, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5815058688, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5817417984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5818106112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5818794240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5819482368, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5820170496, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5820858624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5823217920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5831147776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5831835904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5832524032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5834883328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5837242624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5839601920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5840290048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5842649344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5845008640, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5847367936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5848056064, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5850415360, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5852774656, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5853462784, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5854150912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5854839040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5862768896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5863457024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5864145152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5866504448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5868863744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5869551872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5870240000, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5872599296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5873287424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5873975552, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5874663680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5877022976, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5877711104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5878399232, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5886329088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5887017216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5889376512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5890064640, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5890752768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5891440896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5892129024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5894488320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5896847616, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5899206912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5901566208, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5903925504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5906284800, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5906972928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5909332224, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5911691520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5912379648, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5914738944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5915427072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5916115200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5916803328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5919162624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5927092480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5935022336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5942952192, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5950882048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5953241344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5961171200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5969101056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5971460352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5979390208, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5987320064, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5995249920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6003179776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6011109632, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6011797760, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6019727616, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6027657472, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6035587328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6043517184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6045876480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6053806336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6061736192, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6064095488, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6072025344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6079955200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6087885056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6095814912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6096503040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6104432896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6106792192, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6114722048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6122651904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6130581760, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6131269888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6131958016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6139887872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6140576000, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6148505856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6150865152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6158795008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6166724864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6174654720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6182584576, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6184943872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6192873728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6200803584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6208733440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6216663296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6224593152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6226952448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6227640576, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6235570432, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6237929728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6245859584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6253789440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6261719296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6269649152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6270337280, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6278267136, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6286196992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6288556288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6289244416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6297174272, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6305104128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6313033984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6320963840, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6328893696, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6336823552, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6344753408, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6347112704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6355042560, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6357401856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6359761152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6367691008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6375620864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6376308992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6384238848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6392168704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6394528000, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6396887296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6404817152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6412747008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6420676864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6428606720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6429294848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6437224704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6445154560, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6453084416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6455443712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6457803008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6465732864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6473662720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6474350848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6482280704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6482968832, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6490898688, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6498828544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6506758400, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6514688256, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6522618112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6530547968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6531236096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6539165952, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6539854080, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6547783936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6548472064, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6550831360, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6551519488, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6559449344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6561808640, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6564167936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6572097792, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6580027648, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6587957504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6590316800, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6598246656, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6598934784, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6606864640, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6614794496, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6615482624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6623412480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6631342336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6639272192, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6639960320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6647890176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6655820032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6663749888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6666109184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6666797312, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6667485440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6668173568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6668861696, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6669549824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6670237952, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6672597248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6674956544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6675644672, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6678003968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6680363264, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6682722560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6685081856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6687441152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6688129280, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6690488576, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6691176704, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6691864832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6694224128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6696583424, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6698942720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6706872576, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6707560704, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6708248832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6710608128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6711296256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6713655552, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6721585408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6723944704, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6726304000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6734233856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6736593152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6737281280, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6745211136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6745899264, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6746587392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6747275520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6749634816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6750322944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6758252800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6758940928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6766870784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6767558912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6768247040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6768935168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6769623296, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6771982592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6774341888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6775030016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6777389312, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6778077440, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6786007296, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6788366592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6790725888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6793085184, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6795444480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6796132608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6798491904, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6800851200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6801539328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6803898624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6804586752, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6806946048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6809305344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6811664640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6812352768, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6814712064, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6822641920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6825001216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6832931072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6833619200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6835978496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6836666624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6837354752, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6839714048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6840402176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6841090304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6841778432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6844137728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6852067584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6854426880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6856786176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6859145472, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6859833600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6860521728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6861209856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6869139712, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6869827840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6872187136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6872875264, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6873563392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6875922688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6876610816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6877298944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6879658240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6880346368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6881034496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6883393792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6891323648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 6 } }, { "memory": 6899253504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6907183360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6915113216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6923043072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6930972928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6938902784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6946832640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6954762496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6962692352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6963380480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6964068608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6971998464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6979928320, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6987858176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6995788032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7003717888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7011647744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7012335872, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7014695168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7017054464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7024984320, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7025672448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7033602304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7034290432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7036649728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7039009024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7046938880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7054868736, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7055556864, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7063486720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7065846016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7073775872, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7081705728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7089635584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7090323712, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7098253568, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7098941696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7106871552, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7114801408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7122731264, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7130661120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7138590976, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7146520832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7154450688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7162380544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7164739840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7172669696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7180599552, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7188529408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7189217536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7197147392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7205077248, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7207436544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7215366400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7223296256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7231226112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7239155968, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7247085824, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7247773952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7255703808, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7263633664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7271563520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7279493376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7287423232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7295353088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7303282944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7311212800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7319142656, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7327072512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7335002368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7342932224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7350862080, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7358791936, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7366721792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7369081088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7371440384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7373799680, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7381729536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7389659392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7397589248, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7398277376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7400636672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7408566528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7416496384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7424426240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7432356096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7434715392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7437074688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7439433984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7447363840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7455293696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7463223552, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7471153408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7473512704, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7481442560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7482130688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7490060544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7497990400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7505920256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7513850112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7521779968, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7522468096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7530397952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7531086080, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7533445376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7535804672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7543734528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7551664384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7554023680, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7556382976, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7557071104, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7565000960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7565689088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7568048384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7575978240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7576666368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7579025664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7581384960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7582073088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7590002944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7590691072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7593050368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7593738496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7594426624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7602356480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7604715776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7607075072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7609434368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7610122496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7618052352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7618740480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7621099776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7623459072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7631388928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7632077056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7634436352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7636795648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7639154944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7639843072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7640531200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7641219328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7643578624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7644266752, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7652196608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7660126464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7662485760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7670415616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7672774912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7673463040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7674151168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7682081024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7682769152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7685128448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7687487744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7688175872, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7690535168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7691223296, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7691911424, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7694270720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7694958848, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7702888704, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7703576832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7704264960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7712194816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7714554112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7715242240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7717601536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7725531392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7726219520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7726907648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7727595776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7728283904, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7730643200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7738573056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7740932352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7748862208, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7749550336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7751909632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7754268928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7756628224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7758987520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7759675648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7762034944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7769964800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7770652928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7773012224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7773700352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7774388480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7776747776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7777435904, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7779795200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7780483328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7781171456, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7781859584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7784218880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7786578176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7787266304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7789625600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7791984896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7792673024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7795032320, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7795720448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7796408576, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7798767872, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7801127168, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7803486464, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7805845760, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7806533888, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7807222016, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7809581312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7811940608, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7814299904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7822229760, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7824589056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7825277184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7825965312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7826653440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7827341568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7828029696, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7835959552, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7836647680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7837335808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7839695104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7847624960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7 } }, { "memory": 7855554816, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7863484672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7871414528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7873773824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7881703680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7882391808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7884751104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7885439232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7887798528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7895728384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7898087680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7906017536, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7913947392, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7916306688, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7924236544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7924924672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7927283968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7935213824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7943143680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7945502976, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7953432832, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7961362688, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7963721984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7971651840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7974011136, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7981940992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7989870848, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7997800704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8005730560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8013660416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8014348544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8022278400, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8030208256, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8038138112, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8046067968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8053997824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8061927680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8069857536, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8077787392, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8085717248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8086405376, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8088764672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8089452800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8097382656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8105312512, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8107671808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8115601664, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8123531520, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8131461376, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8139391232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8147321088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8155250944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8163180800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8171110656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8179040512, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8179728640, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8182087936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8184447232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8185135360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8193065216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8200995072, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8208924928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8216854784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8219214080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8219902208, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8227832064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8228520192, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8230879488, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8238809344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8239497472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8247427328, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8255357184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8256045312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8263975168, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8264663296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8272593152, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8280523008, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8288452864, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8289140992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8289829120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8297758976, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8305688832, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8306376960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8307065088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8314994944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8322924800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8325284096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8333213952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8341143808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8341831936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8349761792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8357691648, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8365621504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8373551360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8375910656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8376598784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8377286912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8379646208, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8387576064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8395505920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8397865216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8398553344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8406483200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8408842496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8416772352, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8424702208, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8427061504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8427749632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8435679488, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8443609344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8451539200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8452227328, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8460157184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8468087040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8470446336, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8478376192, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8486306048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8488665344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8489353472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8491712768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8492400896, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8494760192, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8502690048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8503378176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8511308032, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8519237888, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8521597184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8529527040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8530215168, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8532574464, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8540504320, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8541192448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8543551744, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8544239872, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8544928000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8552857856, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8560787712, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8568717568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8571076864, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8571764992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8572453120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8573141248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8575500544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8577859840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8585789696, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8588148992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8596078848, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8604008704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8604696832, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8607056128, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8607744256, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8610103552, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8610791680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8613150976, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8613839104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8614527232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8615215360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8617574656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8625504512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8626192640, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8626880768, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8629240064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8637169920, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8639529216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8641888512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8642576640, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8650506496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8651194624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8653553920, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8655913216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8663843072, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8664531200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8672461056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8674820352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8677179648, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8685109504, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8687468800, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8689828096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8692187392, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8694546688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8702476544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8710406400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8712765696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8713453824, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8721383680, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8729313536, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8731672832, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8734032128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8734720256, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8737079552, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8739438848, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8741798144, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8742486272, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8744845568, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8745533696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8746221824, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8748581120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8749269248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8751628544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8752316672, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8753004800, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8753692928, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8756052224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8756740352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8757428480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8758116608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8758804736, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8759492864, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8761852160, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8764211456, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8766570752, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8767258880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8769618176, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8771977472, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8772665600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8773353728, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8774041856, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8781971712, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8789901568, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8790589696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8792948992, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8793637120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8795996416, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8803926272, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8 } }, { "memory": 8811856128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8 } }, { "memory": 8819785984, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8827715840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8835645696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8843575552, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8851505408, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8853864704, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8856224000, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8856912128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8864841984, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8872771840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8880701696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8888631552, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8896561408, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8904491264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8912421120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8913109248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8913797376, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8921727232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8929657088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8937586944, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8945516800, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8946204928, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8946893056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8949252352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8957182208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8965112064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8973041920, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8980971776, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8981659904, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8982348032, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8990277888, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8998207744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9000567040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9008496896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9016426752, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9024356608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9032286464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9032974592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9035333888, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9043263744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9051193600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9051881728, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9054241024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9062170880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9070100736, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9078030592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9085960448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9093890304, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9101820160, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9109750016, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9117679872, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9125609728, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9133539584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9141469440, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9143828736, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9151758592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9159688448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9167618304, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9168306432, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9176236288, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9176924416, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9179283712, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9187213568, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9195143424, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9203073280, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9211003136, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9211691264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9212379392, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9220309248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9222668544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9230598400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9231286528, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9233645824, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9234333952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9242263808, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9250193664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9258123520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9266053376, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9273983232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9281913088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9284272384, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9292202240, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9300132096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9308061952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9315991808, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9318351104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9319039232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9319727360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9322086656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9330016512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9337946368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9340305664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9340993792, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9343353088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9344041216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9344729344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9345417472, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9347776768, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9348464896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9350824192, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9358754048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9359442176, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9360130304, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9368060160, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9370419456, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9372778752, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9375138048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9377497344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9385427200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9393357056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9401286912, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9403646208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9411576064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9413935360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9414623488, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9422553344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9430483200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9438413056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9440772352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9441460480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9442148608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9450078464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9450766592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9453125888, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9455485184, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9463415040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9471344896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9479274752, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9479962880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9480651008, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9481339136, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9489268992, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9491628288, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9493987584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9494675712, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9495363840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9503293696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9505652992, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9506341120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9507029248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9509388544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9517318400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9525248256, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9533178112, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9541107968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9543467264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9544155392, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9552085248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9554444544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9555132672, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9557491968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9559851264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9567781120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9570140416, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9570828544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9573187840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9573875968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9574564096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9575252224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9577611520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9579970816, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9587900672, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9590259968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9590948096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9591636224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9599566080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9600254208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9602613504, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9603301632, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9611231488, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9613590784, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9615950080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9623879936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9624568064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9626927360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9629286656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9631645952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9632334080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9633022208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9635381504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9643311360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9651241216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9659171072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9659859200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9660547328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9661235456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9663594752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9665954048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9668313344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9669001472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9669689600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9672048896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9674408192, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9676767488, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9679126784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9687056640, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9689415936, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9691775232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9694134528, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9694822656, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9695510784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9696198912, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9698558208, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9700917504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9708847360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9711206656, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9711894784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9712582912, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9720512768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9722872064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9725231360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9725919488, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9726607616, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9727295744, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9735225600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9735913728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9738273024, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9738961152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9741320448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9749250304, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9751609600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9752297728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9760227584, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9 } }, { "memory": 9768157440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9776087296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9776775424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9784705280, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9792635136, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9800564992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9808494848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9816424704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9818784000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9826713856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9834643712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9837003008, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9837691136, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9845620992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9853550848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9855910144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9863840000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9871769856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9874129152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9882059008, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9889988864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9897918720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9905848576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9913778432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9921708288, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9929638144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9931997440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9939927296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9940615424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9941303552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9943662848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9944350976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9952280832, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9960210688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9968140544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9976070400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9978429696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9986359552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9994289408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10002219264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10004578560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10006937856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10014867712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10017227008, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10025156864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10025844992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10033774848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10034462976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10036822272, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10044752128, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10052681984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10060611840, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10062971136, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10070900992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10078830848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10086760704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10094690560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10095378688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10103308544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10111238400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10119168256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10121527552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10129457408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10131816704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10139746560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10142105856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10150035712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10157965568, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10158653696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10166583552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10174513408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10182443264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10190373120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10192732416, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10195091712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10195779840, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10203709696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10211639552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10212327680, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10214686976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10222616832, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10230546688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10232905984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10233594112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10241523968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10242212096, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10250141952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10258071808, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10266001664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10273931520, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10281861376, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10289791232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10290479360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10291167488, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10293526784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10295886080, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10296574208, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10297262336, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10305192192, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10313122048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10315481344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10316169472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10316857600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10319216896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10319905024, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10320593152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10321281280, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10323640576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10325999872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10326688000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10329047296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10331406592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10339336448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10340024576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10342383872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10350313728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10352673024, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10360602880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10368532736, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10370892032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10378821888, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10381181184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10381869312, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10382557440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10383245568, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10385604864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10386292992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10386981120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10389340416, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10390028544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10390716672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10398646528, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10406576384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10414506240, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10416865536, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10424795392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10425483520, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10433413376, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10435772672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10443702528, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10451632384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10452320512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10453008640, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10453696768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10456056064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10458415360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10466345216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10467033344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10467721472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10468409600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10470768896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10471457024, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10479386880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10481746176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10484105472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10486464768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10487152896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10495082752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10497442048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10505371904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10507731200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10510090496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10512449792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10514809088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10522738944, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10523427072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10524115200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10526474496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10528833792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10529521920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10531881216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10532569344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10540499200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10542858496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10543546624, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10545905920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10548265216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10548953344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10556883200, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10557571328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10559930624, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10560618752, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10562978048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10570907904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10578837760, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10586767616, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10594697472, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10602627328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10603315456, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10605674752, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10606362880, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10608722176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10609410304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10610098432, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10610786560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10611474688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10612162816, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10612850944, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10615210240, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10615898368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10623828224, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10631758080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10632446208, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10634805504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10635493632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10637852928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10645782784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10648142080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10656071936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10658431232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10666361088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10667049216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10667737344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10670096640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10678026496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10685956352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10686644480, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10694574336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10695262464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10703192320, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10705551616, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10706239744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10708599040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10716528896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10 } }, { "memory": 10724458752, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10732388608, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10740318464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10748248320, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10756178176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10764108032, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10772037888, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10774397184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10782327040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10790256896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10790945024, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10798874880, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10806804736, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10809164032, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10817093888, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10819453184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10821812480, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10829742336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10837672192, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10845602048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10846290176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10854220032, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10862149888, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10864509184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10872439040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10880368896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10888298752, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10896228608, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10896916736, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10904846592, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10905534720, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10913464576, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10921394432, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10922082560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10930012416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10930700544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10931388672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10939318528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10940006656, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10947936512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10955866368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10963796224, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10971726080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10979655936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10987585792, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10989945088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10997874944, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11000234240, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11000922368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11008852224, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11009540352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11017470208, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11019829504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11022188800, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11022876928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11030806784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11038736640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11046666496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11049025792, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11056955648, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11064885504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11067244800, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11067932928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11075862784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11083792640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11091722496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11099652352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11107582208, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11115512064, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11123441920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11131371776, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11132059904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11139989760, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11147919616, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11148607744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11149295872, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11157225728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11165155584, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11173085440, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11173773568, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11176132864, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11184062720, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11184750848, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11185438976, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11193368832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11201298688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11209228544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11209916672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11217846528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11220205824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11222565120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11223253248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11225612544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11226300672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11228659968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11229348096, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11230036224, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11230724352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11233083648, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11235442944, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11236131072, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11238490368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11240849664, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11241537792, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11242225920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11244585216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11245273344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11253203200, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11255562496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11263492352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11264180480, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11272110336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11274469632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11275157760, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11275845888, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11283775744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11284463872, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11292393728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11293081856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11293769984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11294458112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11295146240, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11297505536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11298193664, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11306123520, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11314053376, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11314741504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11317100800, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11317788928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11318477056, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11320836352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11328766208, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11331125504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11333484800, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11341414656, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11349344512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11351703808, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11354063104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11354751232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11362681088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11370610944, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11378540800, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11380900096, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11383259392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11391189248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11393548544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11395907840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11396595968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11398955264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11401314560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11402002688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11409932544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11417862400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11418550528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11420909824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11423269120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11423957248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11431887104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11439816960, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11442176256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11444535552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11445223680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11447582976, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11455512832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11456200960, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11464130816, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11466490112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11468849408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11469537536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11471896832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11472584960, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11474944256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11477303552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11479662848, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11482022144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11482710272, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11483398400, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11485757696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11488116992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11490476288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11491164416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11493523712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11495883008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11503812864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11511742720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11514102016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11514790144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11522720000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11530649856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11538579712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11546509568, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11548868864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11556798720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11557486848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11558174976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11558863104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11561222400, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11561910528, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11569840384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11577770240, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11580129536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11580817664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11583176960, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11583865088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11591794944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11594154240, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11602084096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11602772224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11603460352, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11611390208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11613749504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11621679360, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11629609216, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11631968512, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11639898368, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11640586496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11641274624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11649204480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11651563776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11659493632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11661852928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11664212224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11664900352, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11672830208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10 } }, { "memory": 11680760064, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11688689920, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11691049216, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11698979072, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11706908928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11714838784, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11715526912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11717886208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11720245504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11720933632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11728863488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11736793344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11744723200, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11745411328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11747770624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11755700480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11763630336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11771560192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11779490048, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11780178176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11782537472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11784896768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11792826624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11800756480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11808686336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11811045632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11818975488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11826905344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11834835200, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11842765056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11850694912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11858624768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11866554624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11874484480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11882414336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11890344192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11898274048, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11906203904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11906892032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11914821888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11917181184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11925111040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11927470336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11935400192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11936088320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11944018176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11944706304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11947065600, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11954995456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11957354752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11965284608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11965972736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11973902592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11981832448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11989762304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11997692160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12005622016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12013551872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12015911168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12018270464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12018958592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12026888448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12029247744, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12037177600, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12045107456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12047466752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12055396608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12057755904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12065685760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12066373888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12067062016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12074991872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12077351168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12085281024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12085969152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12088328448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12089016576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12091375872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12099305728, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12101665024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12109594880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12117524736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12125454592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12126142720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12134072576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12136431872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12144361728, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12146721024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12147409152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12148097280, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12148785408, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12151144704, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12153504000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12154192128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12162121984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12162810112, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12165169408, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12173099264, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12175458560, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12183388416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12185747712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12193677568, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12201607424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12202295552, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12202983680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12203671808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12211601664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12219531520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12227461376, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12229820672, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12230508800, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12231196928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12233556224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12241486080, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12243845376, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12246204672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12248563968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12256493824, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12258853120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12259541248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12260229376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12268159232, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12268847360, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12269535488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12270223616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12270911744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12271599872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12279529728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12287459584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12288147712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12290507008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12292866304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12293554432, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12301484288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12302172416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12304531712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12312461568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12313149696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12321079552, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12323438848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12324126976, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12324815104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12332744960, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12333433088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12334121216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12336480512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12337168640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12337856768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12338544896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12340904192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12343263488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12351193344, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12353552640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12355911936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12363841792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12364529920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12366889216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12369248512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12377178368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12385108224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12393038080, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12395397376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12397756672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12405686528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12406374656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12414304512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12422234368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12422922496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12423610624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12424298752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12424986880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12425675008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12433604864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12435964160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12438323456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12446253312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12448612608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12450971904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12451660032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12454019328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12461949184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12464308480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12466667776, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12467355904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12475285760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12483215616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12483903744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12486263040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12486951168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12494881024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12502810880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12503499008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12504187136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12512116992, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12514476288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12516835584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12517523712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12525453568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12527812864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12535742720, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12543672576, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12544360704, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12545048832, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12552978688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12555337984, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12556026112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12563955968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12564644096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12567003392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12567691520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12570050816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12577980672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12578668800, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12581028096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12583387392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12584075520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12592005376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12592693504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12600623360, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12608553216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12610912512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12618842368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12621201664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12629131520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12637061376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12644991232, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12645679360, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12653609216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12655968512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12663898368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12666257664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12674187520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12682117376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12690047232, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12692406528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12700336384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12701024512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12701712640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12709642496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12710330624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12718260480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12718948608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12726878464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12729237760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12737167616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12745097472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12753027328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12753715456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12754403584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12762333440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12770263296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12770951424, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12778881280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12786811136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12794740992, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12797100288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12805030144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12807389440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12815319296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12816007424, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12823937280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12831867136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12839796992, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12847726848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12848414976, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12856344832, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12864274688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12872204544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12880134400, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12888064256, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12895994112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12898353408, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12906283264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12914213120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12914901248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12915589376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12916277504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12916965632, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12917653760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12925583616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12926271744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12934201600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12936560896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12944490752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12952420608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12954779904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12955468032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12956156160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12964086016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12972015872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12979945728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12982305024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12990234880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12998164736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13006094592, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13014024448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13016383744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13024313600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13025001728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13032931584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13033619712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13035979008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13043908864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13046268160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13054198016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13062127872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13062816000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13070745856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13078675712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13081035008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13083394304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13091324160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13092012288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13092700416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13093388544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13094076672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13102006528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13109936384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13112295680, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13112983808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13120913664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13121601792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13129531648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13137461504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13145391360, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13153321216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13161251072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13161939200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13169869056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13172228352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13180158208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13188088064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13188776192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13189464320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13197394176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13198082304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13206012160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13213942016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13216301312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13224231168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13232161024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13240090880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13242450176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13244809472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13247168768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13249528064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13250216192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13258146048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13258834176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13266764032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13269123328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13277053184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13284983040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13285671168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13293601024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13295960320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13303890176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13311820032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13312508160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13314867456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13315555584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13317914880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13318603008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13320962304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13321650432, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13322338560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13330268416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13332627712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13333315840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13335675136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13338034432, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13340393728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13341081856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13349011712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13351371008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13353730304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13356089600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13356777728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13364707584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13372637440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13374996736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13377356032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13379715328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13382074624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13382762752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13383450880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13384139008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13384827136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13385515264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13387874560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13388562688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13396492544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13397180672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13399539968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13400228096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13402587392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13404946688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13405634816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13406322944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13408682240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13416612096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13418971392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13426901248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13427589376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13428277504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13430636800, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13432996096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13435355392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13437714688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13445644544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13448003840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13450363136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13458292992, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13460652288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13468582144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13470941440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13471629568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13472317696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13480247552, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13482606848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13490536704, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13492896000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13495255296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13497614592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13498302720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13506232576, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13508591872, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13509280000, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13511639296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13513998592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13514686720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13517046016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13524975872, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13527335168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13529694464, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13537624320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13538312448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13539000576, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13539688704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13547618560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13548306688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13556236544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13558595840, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13566525696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13568884992, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13576814848, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13577502976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13585432832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 13 } }, { "memory": 13593362688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13594050816, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13594738944, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13602668800, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13610598656, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13618528512, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13626458368, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13634388224, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13636747520, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13637435648, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13645365504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13653295360, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13661225216, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13669155072, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13677084928, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13685014784, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13692944640, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13700874496, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13703233792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13711163648, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13711851776, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13712539904, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13720469760, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13728399616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13736329472, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13744259328, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13746618624, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13754548480, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13762478336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13770408192, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13771096320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13771784448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13779714304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13782073600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13790003456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13797933312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13805863168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13813793024, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13821722880, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13822411008, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13824770304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13832700160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13840630016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13842989312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13845348608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13853278464, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13861208320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13869138176, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13877068032, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13884997888, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13892927744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13900857600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13903216896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13905576192, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13906264320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13908623616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13909311744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13911671040, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13919600896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13927530752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13935460608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13937819904, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13945749760, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13948109056, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13950468352, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13958398208, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13960757504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13968687360, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13969375488, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13977305344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13979664640, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13982023936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13982712064, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13990641920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13998571776, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14006501632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14007189760, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14015119616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14023049472, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14023737600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14024425728, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14032355584, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14034714880, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14037074176, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14039433472, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14040121600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14048051456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14048739584, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14056669440, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14059028736, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14066958592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14069317888, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14070006016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14072365312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14073053440, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14075412736, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14076100864, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14078460160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14079148288, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14079836416, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14080524544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14081212672, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14089142528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14091501824, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14099431680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14100119808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14102479104, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14103167232, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14103855360, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14104543488, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14106902784, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14109262080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14117191936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14117880064, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14125809920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14126498048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14128857344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14136787200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14137475328, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14145405184, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14153335040, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14161264896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14163624192, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14171554048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14173913344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14181843200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14182531328, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14183219456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14191149312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14193508608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14195867904, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14203797760, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14211727616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14214086912, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14222016768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14222704896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14230634752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14232994048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14240923904, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14243283200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14245642496, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14248001792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14250361088, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14252720384, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14253408512, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14255767808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14256455936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14257144064, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14259503360, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14261862656, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14264221952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14266581248, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14267269376, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14267957504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14268645632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14269333760, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14270021888, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14270710016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14278639872, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14286569728, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14294499584, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14296858880, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14297547008, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14299906304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14300594432, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14301282560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14303641856, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14306001152, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14313931008, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14316290304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14316978432, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14317666560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14325596416, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14327955712, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14335885568, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14336573696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14338932992, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14346862848, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14347550976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14349910272, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14352269568, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14352957696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14360887552, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14361575680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14362263808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14370193664, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14370881792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14373241088, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14373929216, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14381859072, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14389788928, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14392148224, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14400078080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14400766208, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14401454336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14409384192, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14410072320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14410760448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14418690304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14426620160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14434550016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14442479872, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14444839168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14447198464, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14455128320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14455816448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14456504576, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14464434432, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14466793728, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14469153024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14477082880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14485012736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14492942592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14495301888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14497661184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14498349312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14499037440, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14506967296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14514897152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14517256448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14525186304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14525874432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14533804288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14541734144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14549664000, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14557593856, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14558281984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14558970112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14561329408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14569259264, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14577189120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14585118976, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14585807104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14588166400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14588854528, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14596784384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14604714240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14607073536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14615003392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14622933248, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14630863104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14638792960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14646722816, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14654652672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14655340800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14663270656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14671200512, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14671888640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14679818496, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14687748352, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14695678208, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14696366336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14704296192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14712226048, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14720155904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14722515200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14730445056, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14738374912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14746304768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14754234624, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14754922752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14762852608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14763540736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14764228864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14772158720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14780088576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14782447872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14790377728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14792737024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14800666880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14801355008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14803714304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14804402432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14812332288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14820262144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14822621440, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14830551296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14838481152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14840840448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14848770304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14856700160, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14859059456, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14861418752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14862106880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14862795008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14865154304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14865842432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14873772288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14874460416, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14875148544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14883078400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14883766528, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14891696384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14899626240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14900314368, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14901002496, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14908932352, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14916862208, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14919221504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14919909632, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14927839488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14935769344, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14936457472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14944387328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14952317184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14954676480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14962606336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14964965632, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14972895488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14973583616, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14975942912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14976631040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14978990336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14979678464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14982037760, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14982725888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14983414016, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14985773312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14988132608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14990491904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14992851200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15000781056, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15003140352, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15003828480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15004516608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15005204736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15007564032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15015493888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15017853184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15020212480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15020900608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15023259904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15025619200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15026307328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15034237184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15036596480, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15044526336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15046885632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15054815488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15055503616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15056191744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15064121600, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15072051456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15079981312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15087911168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15095841024, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15103770880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15106130176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15114060032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15121989888, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15124349184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15126708480, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15127396608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15135326464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15137685760, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15145615616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15146303744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15146991872, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15147680000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15148368128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15149056256, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15156986112, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15157674240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15160033536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15167963392, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15175893248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15183823104, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15191752960, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15194112256, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15194800384, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15202730240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15203418368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15211348224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15213707520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15221637376, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15223996672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15224684800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15225372928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15226061056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15228420352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15230779648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15233138944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15241068800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15241756928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15242445056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15244804352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15245492480, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15247851776, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15248539904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15249228032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15251587328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15252275456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15254634752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15255322880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15257682176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15265612032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15267971328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15275901184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15278260480, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15280619776, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15288549632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15296479488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15297167616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15299526912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15301886208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15309816064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15310504192, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15318434048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15326363904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15334293760, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15334981888, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15335670016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15338029312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15340388608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15342747904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15345107200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15345795328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15346483456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15354413312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15356772608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15357460736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15365390592, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15367749888, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15370109184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15372468480, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15380398336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15388328192, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15396258048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15396946176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15399305472, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15407235328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15407923456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15410282752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15410970880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15418900736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15421260032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15429189888, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15437119744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15439479040, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15447408896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15448097024, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15456026880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15463956736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15471886592, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15479816448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15482175744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15490105600, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15498035456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15505965312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15513895168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15521825024, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15529754880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15537684736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15545614592, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15546302720, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15546990848, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15554920704, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15555608832, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15563538688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15564226816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15572156672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15572844800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15580774656, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15583133952, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15583822080, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15591751936, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15599681792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15607611648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15608299776, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15616229632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15624159488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15624847616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15632777472, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15635136768, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15643066624, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15645425920, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15653355776, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15655715072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15663644928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15664333056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15665021184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15672951040, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15680880896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15683240192, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15691170048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15693529344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15701459200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15703818496, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15711748352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15719678208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15722037504, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15729967360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15730655488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15731343616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15732031744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15739961600, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15742320896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15750250752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15758180608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15766110464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15774040320, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15781970176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15789900032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15792259328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15800189184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15802548480, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15810478336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15811166464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15819096320, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15827026176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15834956032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15835644160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15838003456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15845933312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15853863168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15854551296, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15855239424, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15855927552, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15856615680, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15857303808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15865233664, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15867592960, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15875522816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15883452672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15891382528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15899312384, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15900000512, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15902359808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15904719104, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15905407232, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15906095360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15908454656, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15916384512, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15924314368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15925002496, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15927361792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15929721088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15932080384, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15940010240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15940698368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15948628224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15950987520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15953346816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15961276672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15963635968, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15964324096, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15966683392, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15974613248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15982543104, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15990472960, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15998402816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16006332672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16014262528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16016621824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16018981120, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16026910976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16027599104, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16035528960, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16043458816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16051388672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16059318528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16067248384, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16067936512, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16070295808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16070983936, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16078913792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16079601920, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16080290048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16082649344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16085008640, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16092938496, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16095297792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16095985920, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16096674048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16099033344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16106963200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16109322496, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16111681792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16112369920, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16114729216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16115417344, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16117776640, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16125706496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16126394624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16134324480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16136683776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16139043072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16139731200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16142090496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16142778624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16145137920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16145826048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16148185344, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16156115200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16164045056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16171974912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16172663040, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16173351168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16175710464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16176398592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16177086720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16177774848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16178462976, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16180822272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16183181568, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16191111424, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16193470720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16201400576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16209330432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16217260288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16219619584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16221978880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16224338176, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16232268032, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16232956160, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16240886016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16248815872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16251175168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16253534464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16254222592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16262152448, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16264511744, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16272441600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16274800896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16275489024, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16283418880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16284107008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16284795136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16285483264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16286171392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16286859520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16289218816, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16297148672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16305078528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16305766656, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16313696512, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16316055808, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16323985664, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16331915520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16334274816, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16336634112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16338993408, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16339681536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16340369664, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16348299520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16348987648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16356917504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16357605632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16359964928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16362324224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16364683520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16365371648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16367730944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16370090240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16372449536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16380379392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16388309248, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16390668544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16391356672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16392044800, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16394404096, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16395092224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16395780352, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16403710208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16411640064, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16419569920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16421929216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16422617344, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16430547200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16438477056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16446406912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16454336768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16455024896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16462954752, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16463642880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16471572736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16479502592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16481861888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16489791744, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16497721600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16498409728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16499097856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16507027712, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16507715840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16508403968, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16509092096, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16509780224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16517710080, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16520069376, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16527999232, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16528687360, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16536617216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16544547072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16552476928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16553165056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16561094912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16569024768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16571384064, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16579313920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16581673216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16589603072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16597532928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16605462784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16606150912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16608510208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16616440064, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16617128192, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16619487488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16620175616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16628105472, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16636035328, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16643965184, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16646324480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16654254336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16662184192, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16664543488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16665231616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16667590912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16675520768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16683450624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16691380480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16699310336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16707240192, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16715170048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16723099904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16723788032, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16731717888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16732406016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16734765312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16737124608, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16745054464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16752984320, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16760914176, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16761602304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16769532160, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16770220288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16778150144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16778838272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16779526400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16787456256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16789815552, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16792174848, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16800104704, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16802464000, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16803152128, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16811081984, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16819011840, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16826941696, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16829300992, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16837230848, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16837918976, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16845848832, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16848208128, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16856137984, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16856826112, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16859185408, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16867115264, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16867803392, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16875733248, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16883663104, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16891592960, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16892281088, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16892969216, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16893657344, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16896016640, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16903946496, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16911876352, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16912564480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16920494336, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16928424192, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16936354048, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16944283904, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16944972032, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16952901888, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16960831744, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16963191040, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16963879168, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16964567296, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16972497152, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16973185280, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16973873408, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16981803264, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16982491392, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16990421248, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 16998351104, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17006280960, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17008640256, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17009328384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17011687680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17014046976, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17021976832, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17024336128, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17025024256, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17032954112, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17033642240, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17034330368, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17036689664, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17037377792, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17039737088, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17040425216, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17048355072, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17049043200, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17049731328, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17052090624, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17054449920, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17062379776, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16 } }, { "memory": 17070309632, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17078239488, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17080598784, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17088528640, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17089216768, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17091576064, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17099505920, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17101865216, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17102553344, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17103241472, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17105600768, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17107960064, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17108648192, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17109336320, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17110024448, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17112383744, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17114743040, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17122672896, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17125032192, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17127391488, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17129750784, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17132110080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17132798208, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17135157504, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17143087360, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17143775488, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17146134784, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17148494080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17150853376, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17158783232, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17161142528, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17163501824, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17171431680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17172119808, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17180049664, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17180737792, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17181425920, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17183785216, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17191715072, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17192403200, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17194762496, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17202692352, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17210622208, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17218552064, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17219240192, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17221599488, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17229529344, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17231888640, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17239818496, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17247748352, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17248436480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17250795776, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17258725632, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17261084928, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17269014784, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17271374080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17273733376, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17276092672, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17284022528, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17291952384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17299882240, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17300570368, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17308500224, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17316430080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17317118208, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17319477504, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17321836800, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17329766656, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17330454784, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17338384640, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17340743936, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17343103232, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17345462528, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17347821824, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17348509952, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17350869248, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17353228544, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17355587840, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17357947136, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17358635264, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17366565120, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17367253248, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17375183104, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17377542400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17378230528, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17378918656, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17386848512, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17394778368, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17402708224, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } } ]